package com.yym.cloudweb.crawl.job;

import com.yym.cloudweb.crawl.station.jd.JdGpuPageProcessor;
import com.yym.cloudweb.crawl.station.jd.JdGpuPipeline;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

@Service("jdGoodJob")
public class JdGoodJob {

    @Autowired
    JdGpuPipeline jdGpuPipeline;

    public void execute(){
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("10.27.199.7",3128)));
        Spider spider = Spider.create(new JdGpuPageProcessor());
        spider.addPipeline(jdGpuPipeline);
        spider.addUrl("https://search.jd.com/Search?keyword=显卡&wq=显卡&page=1&click=0");
        spider.setDownloader(httpClientDownloader);
        spider.thread(2).run();
    }

}
